{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Logistic Regression.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "r25onCT8olDj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QUMwBj9GolDx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset=pd.read_csv(\"Haberman Dataset.csv\");\n",
        "X=dataset.iloc[:,:3];\n",
        "y=dataset.iloc[:,3:];"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2fpg-C2FolD5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.cross_validation import train_test_split\n",
        "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,stratify=y);\n",
        "y_train.Survival.value_counts()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "E2jq9_BKolEL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "lr=LogisticRegression();\n",
        "lr.fit(X_train,y_train.values.ravel())\n",
        "print(lr.intercept_, lr.coef_)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "x0MZna1HolEU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lr.predict(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_2Rggr-polEg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lr.score(X_test,y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GSYXJykcolEo",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# What if we use regression algorithm for classification dataset"
      ]
    },
    {
      "metadata": {
        "id": "tnaHOPmlolEp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LinearRegression\n",
        "lr=LinearRegression();\n",
        "lr.fit(X_train,y_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pG6Cg94kolE1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lr.predict(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DikiDef3olFJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lr.score(X_test,y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MxHEHz15olFX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "B8oDb_DmolFj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%matplotlib inline\n",
        "dataset.plot.scatter('Positive auxilary-nodes', 'Survival')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MFIdQbsColFy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset.plot.scatter('Age', 'Survival')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JiAqidueolGA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset.plot.scatter('Year', 'Survival')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qm40QTmzolGW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y.shape\n",
        "X.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9ztruY_OolGj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "sns.regplot(x='Year', y='Survival', data=dataset)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "L8_OSqURolGt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from pydoc import help\n",
        "import numpy as np\n",
        "pc=np.corrcoef(X_train,y_train,rowvar=False)\n",
        "print(pc)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-nZ8dGKVolG-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestClassifier \n",
        "rfc=RandomForestClassifier()\n",
        "rfc.fit(X_train,y_train.values.ravel())\n",
        "feature_importance = rfc.feature_importances_\n",
        "feature_importance"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9EdgOtQ9olHL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset.iloc[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3O_hL_0yolHY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.figure(figsize=(16, 6))\n",
        "plt.yscale('log', nonposy='clip')\n",
        "plt.bar(range(len(feature_importance)), feature_importance, align='center')\n",
        "plt.xticks(range(len(feature_importance)),dataset.columns, rotation='vertical')\n",
        "plt.title('Feature importance')\n",
        "plt.ylabel('Importance')\n",
        "plt.xlabel('Features')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tMOnIy3oolHh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "feature_importances = pd.DataFrame(rfc.feature_importances_, columns=['weights'], index=X_train.columns)\n",
        "feature_importances"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GZVqSn6oolHn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def extract_pruned_features(feature_importances, min_score=0.4):\n",
        "    column_slice = feature_importances[feature_importances['weights'] > min_score]\n",
        "    return column_slice.index.values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AwTtSmJkolHt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "pruned_featurse = extract_pruned_features(feature_importances, min_score=0.32)\n",
        "print('Pruned features:', pruned_featurse)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4ub_322qolHy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X_train_reduced = X_train[pruned_featurse]\n",
        "X_test_reduced = X_test[pruned_featurse]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YYETT3UColH4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def fit_and_print_metrics(X_train, y_train, X_test, y_test, rfc):\n",
        "    rfc.fit(X_train, y_train.values.ravel())\n",
        "\n",
        "    predictions_proba = model.predict_proba(X_test)\n",
        "    predictions = rfc.predict(X_test)\n",
        "\n",
        "    log_loss_score = sklearn.metrics.log_loss(y_test, predictions_proba)\n",
        "    acc = sklearn.metrics.accuracy_score(y_test, predictions)\n",
        "    f1 = sklearn.metrics.f1_score(y_test, predictions)\n",
        "\n",
        "    print('Log loss: %.5f' % log_loss_score)\n",
        "    print('Acc: %.5f' % acc)\n",
        "    print('F1: %.5f' % f1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PjJHwLsFolID",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import sklearn\n",
        "model = RandomForestClassifier(50, n_jobs=8)\n",
        "fit_and_print_metrics(X_train_reduced, y_train, X_test_reduced, y_test, model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AJqgdig5olIJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = RandomForestClassifier(50, n_jobs=8)\n",
        "fit_and_print_metrics(X_train, y_train, X_test, y_test, model)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}